Problem # 1¶Given an Image, identify sections where there is no product present. Preferred library OpenCV
Steps taken to get the empty space as foreground
import cv2 #cv2 v4.2.0
import glob
import csv,os
import argparse
import numpy as np
from scipy import ndimage #scipy v1.5.1
import matplotlib.pyplot as plt
from skimage.feature import peak_local_max #skimage v0.17.2
from skimage.morphology import watershed
import tensorflow
tensorflow.__version__
from skimage.color import rgb2gray
from skimage.feature import blob_dog, blob_doh
from keras.models import model_from_json #keras v 2.1.5 default for Tensorflow 2.2.0
images = os.listdir(os.getcwd() + "/retail_images/")
escape=False
def get_foreground(img):
'''
input : img name
output : Image with Blank space as foreground and rest as background
'''
h_kernel = np.ones((5,3),np.uint8)
kernal = np.ones((3,3),np.uint8)
if escape==True:
cv2.destroyAllWindows()
# break
else:
#Read the Image in color format
color=cv2.imread(os.getcwd() + "/retail_images/" + img.split(".")[0]+".jpg")
gray = cv2.cvtColor(color, cv2.COLOR_BGR2GRAY)
#Threshold the image using Otsu
ret3,th3 = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
# Create the minimum and maximum threshold values for Canny Edge detection
hist_eq =cv2.equalizeHist(gray)
min_thresh=0.66*np.median(hist_eq)
max_thresh=1.33*np.median(hist_eq)
edge_1 = cv2.Canny(gray,min_thresh,max_thresh)
# Dilate the edges to make it more visible
edge_2 = cv2.dilate(edge_1,None,iterations = 1)
image_th_inv = cv2.bitwise_not(cv2.dilate(edge_2,h_kernel,iterations = 2))
return image_th_inv,gray,ret3
def combine_bounding_boxes(upp_th,blob_list_sorted):
'''
upp_th uppr threshold for bbx coordinates difference
blob_list_sorted : Lsit of blobs whoch needs to be arranged to remove intersecting and overlapping bbx
return : new bounding boxes list
'''
for i,val in enumerate(range(0,len(blob_list_sorted))):
for j in range(i+1,len(blob_list_sorted)-2):
if max(i,j) < len(blob_list_sorted) and max([abs(np.diff(x)) for x in zip(blob_list_sorted[j][0:2], blob_list_sorted[i][0:2])]) < upp_th:
blob_list_sorted = blob_list_sorted+[[max(x) for x in zip(blob_list_sorted[j], blob_list_sorted[i])][0:2]+[blob_list_sorted[j][2]+blob_list_sorted[i][2]]]
del blob_list_sorted[i]
del blob_list_sorted[j]
len_val = len(blob_list_sorted)
return blob_list_sorted
def show_segmentation(img):
'''
input: image name
output : Bounding boxes for the empty space where thre is no product present
'''
original = cv2.imread(os.getcwd() + "/retail_images/" + img.split(".")[0]+".jpg")
image_th_inv,gray,th = get_foreground(img)
upp_th = 4/(original.shape[0]*original.shape[0]); count = 1
image_th_inv[image_th_inv > th] = 255
image_th_inv[image_th_inv < th] = 0
_image = cv2.erode(image_th_inv,np.ones((3,3),np.uint8),iterations = 1)
fig, axes = plt.subplots(1, 2, figsize=(12, 6))#, sharex=True, sharey=True)
ax = axes.ravel()
ax[0].imshow(cv2.cvtColor(original,cv2.COLOR_RGB2BGR))
ax[1].imshow(_image)
plt.show()
import warnings
warnings.simplefilter("ignore")
def main(img):
'''
input: image name
output : Bounding boxes for the empty space where thre is no product present
'''
results = {"bbox" : [], "coords" : []}
original = cv2.imread(os.getcwd() + "/retail_images/" + img.split(".")[0]+".jpg")
image_th_inv,gray,th = get_foreground(img)
upp_th = 4/(original.shape[0]*original.shape[0]); count = 1
image_th_inv[image_th_inv > th] = 255
image_th_inv[image_th_inv < th] = 0
_image = cv2.erode(image_th_inv,np.ones((3,3),np.uint8),iterations = 1)
blobs_dog = blob_dog(_image, max_sigma=40, threshold=.10)
blob_list_sorted = sorted([[int(blob[0]), int(blob[1]),int(blob[2])] for blob in blobs_dog])
blob_list_sorted = combine_bounding_boxes(upp_th,blob_list_sorted)
try:
for i,blob in enumerate(blob_list_sorted):
y, x, r = blob
if 6 < r :
original = cv2.rectangle(original,(int(x),int(y)),(int((x+r)),int((y+r))),(255,0,0),2)
cv2.putText(original,str(count),(int(x+r/2),int(y+r/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(0,0,255),2)
count = count+1
results["bbox"].append(count)
results["coords"].append([x,y,x+r,y+r])
fig, axes = plt.subplots(1, 2, figsize=(20, 12))#, sharex=True, sharey=True)
ax = axes.ravel()
ax[0].imshow(cv2.imread("retail_images/" + img.split(".")[0]+".jpg"))
ax[1].imshow(original)
plt.show()
except:pass
return results
## Show segmentation
[show_segmentation(img) for img in images]
## Show Bounding Boxes
[print(main(img)) for img in images]
For seperating objects, CascadeClassifier can be trained for various possible objects in the image for seperating objects of interest vs other objects.
Solving the above problem with the mean of AI/ML based approaches.
Pls note, objective of the following approach is limied to solving the problem described and deployment of the solution in real time detection and pushing notifications to the end user UI is beyond the scope of this notebook.
Objective : Detection of blank spaces in the product shelves.
a. Annotated data which has bounding boxes for void/blank spaces where there is no product placed.
b. Classes names under which shelf products can fall under.
This information will be required for machine to automatically identify relative dimensions and specifications for blak/void space.
c. Data across various departments for better generalisation.
A GPU or i7/i9 processor base CPU for model training.
1. Train a VGG 16/19/UNet or sliding window deep elarning models using respective architectures to only train last few layers on annotated data if context in the images is almost constant.
This means training seperate models for diiferent SKUs will serve the purpose.
2. The other approach will be to use YOLO/imagenet which are pre-trained models for identifying objects in different contexts.
Step by step solution can be documented post applying the above deep learning models in next Notebook.
Scenario 1: Idenfication blank spaces in middle of the retail shelves will require a lot of contexual information to seggregate objects from blank spaces.
Scenario 2. If images are warped or has angle, then pixel depth information will go for a try in identifying features for blank/void space.
Scenario 3. Different light conditions will bias the information.
Scenario 4. Images with lot of surrounding information like roof, floor, different other objects palced near sheves will add error in the prediction.
Scenario 5. Images with clear frames and only has images of shelf products are best suited for model training.
Scenario 6. Images with various size items in the shelf will add challanges in finding true reference for the blank/void space in the shelf.
Scenario 7. Shelves have different patterns in their designs.